

 
"""
Purpose: Bulk/Batch Querying and Retrieval of Clinical (Trial) Research Publications from Elsevier Scopus Literature Database

Author : H. Lin, Ph.D., https://orcid.org/0000-0003-4060-7336 

This script contained Python programming codes

Version: Created on 23 rd May 2025, and updated on 21st Aug. 2025 (Lastest)

Note: Scopus API for bulk querying and download is not free. It requires purchasing or institutional subscription payment.

"""



# Python. Import modules required
import requests
import json
import time

def search_scopus_by_nctid_batch(nct_ids, api_key, inst_token=None, max_results_per_call=25):
    """
    nct_ids (NCT IDs) are the identifier codes of clinical trials from the database clinicaltrials.gov.
    Searches Scopus for papers related to a list of NCT IDs.
    Requires an Elsevier Scopus API key and potentially an institutional token.

    Args:
        nct_ids (list): List of NCT ID strings (e.g., ['NCT000001', 'NCT000002']).
        api_key (str): Your Elsevier Scopus API key.
        inst_token (str, optional): Your institutional token if required by Elsevier. Defaults to None.
        max_results_per_call (int): Max number of results per API call (Scopus limit often 25 or 200).

    Returns:
        dict: Key is NCT ID, value is a list of found papers (metadata).
    """

    if not nct_ids:
        print("NCT ID list is empty.")
        return {}

    base_url = "https://api.elsevier.com/content/search/scopus"
    
    headers = {
        "Accept": "application/json",
        "X-ELS-APIKey": api_key
    }
    if inst_token:
        headers["X-ELS-Insttoken"] = inst_token

    found_papers = {nctid: [] for nctid in nct_ids}
    
    # Scopus API often handles multiple OR queries well, but constructing a query
    # for 1000 NCT IDs might hit URL length limits or query complexity limits.
    # It's safer to chunk the NCT IDs.
    chunk_size_nctid = 50 # Process 50 NCT IDs per query

    print(f"Starting Scopus search for {len(nct_ids)} NCT IDs...")

    for i in range(0, len(nct_ids), chunk_size_nctid):
        nct_id_chunk = nct_ids[i:i + chunk_size_nctid]
        
        # Scopus search field for clinical trial numbers is 'DO' or in 'TITLE-ABS-KEY'
        # Check Scopus API documentation for the most precise field.
        # Often, a search like '("NCTxxxxxxx")' in 'TITLE-ABS-KEY' is effective.
        query_parts = [f'TITLE-ABS-KEY("{nctid}")' for nctid in nct_id_chunk]
        query_string = " OR ".join(query_parts)

        params = {
            "query": query_string,
            "count": max_results_per_call,
            "start": 0 
        }
        
        print(f"\nProcessing Scopus batch {i+1}-{min(i+chunk_size_nctid, len(nct_ids))}...")
        print(f"Query: {query_string[:150]}...") # Show part of the query

        total_results = float('inf')
        current_start = 0

        while current_start < total_results:
            params["start"] = current_start
            
            try:
                response = requests.get(base_url, headers=headers, params=params, timeout=60)
                response.raise_for_status() # Raise an HTTPError for bad responses (4xx or 5xx)

                data = response.json()
                
                # Check for results and pagination info
                if 'search-results' in data and 'opensearch:totalResults' in data['search-results']:
                    total_results = int(data['search-results']['opensearch:totalResults'])
                else:
                    total_results = 0 # No results or unexpected format

                if 'entry' in data['search-results'] and data['search-results']['entry']:
                    for entry in data['search-results']['entry']:
                        paper_info = {
                            "title": entry.get('dc:title', 'N/A'),
                            "authors": [auth['given-name'] + " " + auth['surname'] for auth in entry.get('author', []) if 'given-name' in auth and 'surname' in auth],
                            "journal": entry.get('prism:publicationName', 'N/A'),
                            "doi": entry.get('prism:doi', 'N/A'),
                            "pii": entry.get('pii', 'N/A'), # Elsevier's unique identifier
                            "eid": entry.get('eid', 'N/A'), # Elsevier ID
                            "link": next((link['@href'] for link in entry.get('link', []) if link.get('@rel') == 'abstract'), 'N/A')
                        }
                        
                        # Attempt to associate with original NCT ID (rough method)
                        entry_text = f"{paper_info['title']} {entry.get('prism:url', '')}" # Simplified
                        for original_nctid in nct_id_chunk:
                            if original_nctid.lower() in entry_text.lower(): # Check if NCT ID is in the title or URL
                                found_papers[original_nctid].append(paper_info)
                                break # Add to the first matching NCT ID and stop checking this paper

                    current_start += len(data['search-results']['entry'])
                    # Add a small delay to respect rate limits
                    time.sleep(0.1) 
                else:
                    break # No more entries
            
            except requests.exceptions.RequestException as e:
                print(f"Scopus API Request failed: {e}")
                if 'response' in locals() and hasattr(response, 'text'):
                    print(f"Response content: {response.text}")
                break
            except json.JSONDecodeError as e:
                print(f"Scopus JSON parsing error: {e}")
                if 'response' in locals() and hasattr(response, 'text'):
                    print(f"Response content: {response.text}")
                break

    return found_papers


# --- Usage Example (Conceptual - Requires your API Key) ---
your_nct_ids_list = [f"NCT{str(i).zfill(7)}" for i in range(1, 100)] # Using 100 for example
your_scopus_api_key = "YOUR_SCOPUS_API_KEY" # <<< REPLACE THIS, using the registered and applied API key from the Elesiver Developer Portal: dev.elsevier.com(/apikey/manage)
your_scopus_inst_token = None # <<< REPLACE if your institution requires one

if your_scopus_api_key == "YOUR_SCOPUS_API_KEY":
    print("\nWARNING: Please replace 'YOUR_SCOPUS_API_KEY' with your actual Scopus API key.")
    print("Scopus API is not free and requires an institutional subscription and API key.")
else:
    scopus_results = search_scopus_by_nctid_batch(your_nct_ids_list, your_scopus_api_key, your_scopus_inst_token)

    print("\n--- Scopus Search Complete ---")
    total_found_scopus_papers = sum(len(papers) for papers in scopus_results.values())
    print(f"Total found papers: {total_found_scopus_papers} (may contain duplicates/approximate matches).")

    # Print some example results
    count_printed_scopus = 0
    for nctid, papers in scopus_results.items():
        if papers:
            print(f"\nNCT ID: {nctid}")
            for paper in papers[:2]: # Print up to 2 papers per NCT ID as example
                print(f"  - Title: {paper['title']}")
                print(f"    Journal: {paper['journal']}")
                print(f"    DOI: {paper['doi']}")
            count_printed_scopus += 1
            if count_printed_scopus >= 5:
                break